package com.chance.cc.crawler.development.bootstrap.taobao;

import com.alibaba.fastjson.JSON;
import com.chance.cc.crawler.core.CrawlerEnum;
import com.chance.cc.crawler.core.downloader.HttpConstant;
import com.chance.cc.crawler.core.downloader.HttpRequestBody;
import com.chance.cc.crawler.core.downloader.proxy.Proxy;
import com.chance.cc.crawler.core.filter.FilterUtils;
import com.chance.cc.crawler.core.record.CrawlerRequestRecord;
import com.chance.cc.crawler.development.controller.DevCrawlerController;
import com.chance.cc.crawler.development.utils.RedisReader;
import org.apache.commons.lang3.StringUtils;

import static com.chance.cc.crawler.core.CrawlerEnum.CrawlerDataType.comment;
import static com.chance.cc.crawler.core.CrawlerEnum.CrawlerRequestType.*;

/**
 * @author bx
 * @date 2020/11/11 0011 10:44
 */
public class TaobaoCommentStart {

    public static final String domainId = "taobao";

    private static Proxy proxy = new Proxy();
    static {
        //代理配置
        proxy.setHost("http-dyn.abuyun.com");
        proxy.setPort(9020);
        proxy.setUsername("HEW657EL99F83S9D");
        proxy.setPassword("8916B1F3F10B1979");
    }

    public static void main(String[] args) {
        dev();
//        prod();

    }

    private static void prod(){
        CrawlerRequestRecord requestRecord = CrawlerRequestRecord.builder()
                .startPageRequest(domainId, CrawlerEnum.CrawlerRequestType.turnPage)
                .httpUrl("https://rate.taobao.com/feedRateList.htm")
                .releaseTime(System.currentTimeMillis())
                .needWashed(false)
                .needParsed(false)
                .resultLabelTag(comment)
                .filter(CrawlerEnum.CrawlerRecordFilter.keyOrDateRange)
                .addFilterInfo(FilterUtils.memoryFilterKeyInfo(domainId))
                .addFilterInfo(FilterUtils.dateRangeFilterInfo(24*1,null))
                .proxy(proxy)
                .build();
        requestRecord.setDownload(false);
        requestRecord.setSkipPipeline(true);

        //初始 comment crawler request record
        String requestQueueName = StringUtils.joinWith("-","crawler",domainId,"comment","queue");
        CrawlerRequestRecord initCommentCrawlerRecord = CrawlerRequestRecord.builder()
                .startPageRequest("taobao_comment_keywords",turnPageItem)
                .httpUrl("http://localhost:9599/crawler/domain/common/api/v1/taobao/results/queue/init?requestQueueName="+requestQueueName)
                .requestLabelTag(supportSource)
                .requestLabelTag(internalDownload)
                .build();
        HttpRequestBody jsonBody = HttpRequestBody.json(JSON.toJSONString(requestRecord), "utf-8");
        initCommentCrawlerRecord.getHttpRequest().setMethod(HttpConstant.Method.POST);
        initCommentCrawlerRecord.getHttpRequest().setRequestBody(jsonBody);

        //user agents source record
        CrawlerRequestRecord userAgentsCrawlerRecord = CrawlerRequestRecord.builder()
                .startPageRequest("taobao_comment_user_agents", turnPageItem)
                .httpUrl("https://fake-useragent.herokuapp.com/browsers/0.1.11")
                .requestLabelTag(supportSource)
                .requestLabelTag(internalDownload)
                .build();

        DevCrawlerController.builder()
                .crawlerThreadNum(10)
                .triggerInfo(domainId,domainId,System.currentTimeMillis(),domainId)
                .crawlerRequestQueue(DevCrawlerController.devRequestQueue(requestQueueName))
                .consoleResultPipeline()
                .fileResultPipeline("D:\\chance\\log\\taobao\\taobao_comment.log",false)
                .requestRecord(requestRecord)
//                .supportRecord(userAgentsCrawlerRecord)
                .supportRecord(initCommentCrawlerRecord)
                .build().start();
    }

    private static void dev(){
        RedisReader redisReader = new RedisReader("192.168.1.215",6379,4);
        CrawlerRequestRecord requestRecord = redisReader.recordPop(
                "crawler-taobao-product-detail-queue_list",
                "crawler-taobao-product-detail-queue_map");

        DevCrawlerController.builder()
                .crawlerThreadNum(1)
                .triggerInfo(domainId,domainId,System.currentTimeMillis(),domainId)
                .crawlerRequestQueue(DevCrawlerController.devRequestQueue("crawler-taobao-comment-queue"))
                .consoleResultPipeline()
                .fileResultPipeline("D:\\chance\\log\\taobao\\taobao_comment.log",false)
                .requestRecord(requestRecord)
                .build().start();
    }
}
